{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/laz4rz/miniconda3/envs/gpt/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading gpt2 model weights from transformers\n",
      "   Loading: transformer.wte.weight\n",
      "     from: torch.Size([50257, 768])\n",
      "       to: torch.Size([50257, 768])\n",
      "   Loading: transformer.wpe.weight\n",
      "     from: torch.Size([1024, 768])\n",
      "       to: torch.Size([1024, 768])\n",
      "   Loading: transformer.h.0.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.0.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.0.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.0.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.0.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.0.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.0.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.1.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.1.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.1.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.1.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.1.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.1.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.2.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.2.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.2.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.2.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.2.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.2.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.3.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.3.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.3.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.3.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.3.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.3.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.4.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.4.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.4.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.4.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.4.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.4.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.5.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.5.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.5.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.5.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.5.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.5.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.6.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.6.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.6.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.6.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.6.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.6.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.7.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.7.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.7.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.7.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.7.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.7.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.8.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.8.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.8.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.8.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.8.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.8.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.9.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.9.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.9.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.9.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.9.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.9.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.10.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.10.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.10.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.10.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.10.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.10.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.attn.c_attn.weight\n",
      "     from: torch.Size([768, 2304])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.11.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.11.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.11.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.mlp.c_fc.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.11.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.11.mlp.c_proj.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.11.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.ln_f.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.ln_f.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: lm_head.weight\n",
      "     from: torch.Size([50257, 768])\n",
      "       to: torch.Size([50257, 768])\n",
      "> Hello, I'm a language model, not a programming platform! I just make decisions based on other projects. I try to do that.\"\n",
      "\n",
      "\n",
      "> Hello, I'm a language model, a kind of a \"first class citizen\" of the world and a person that comes from a much more egalitarian\n",
      "> Hello, I'm a language model, and I'm starting to talk about the notion of the syntax, and I'm also working on an extension that\n",
      "> Hello, I'm a language model, because I'm writing real-time. I'm writing all languages. And I'm working with languages for me\n",
      "> Hello, I'm a language model, I don't know where to begin but I know there is a big deal going on with our society. What\n"
     ]
    }
   ],
   "source": [
    "from gpt2 import GPT\n",
    "\n",
    "model = GPT.from_pretrained('gpt2', verbose=True)\n",
    "num_return_sequences = 5\n",
    "max_length = 30\n",
    "\n",
    "model.generate(\"Hello, I'm a language model,\", max_length=max_length, num_return_sequences=num_return_sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading GPT-2 model weights from local file: models/shakespear_500.pth\n",
      "   Loading: transformer.wte.weight\n",
      "     from: torch.Size([50304, 768])\n",
      "       to: torch.Size([50304, 768])\n",
      "   Loading: transformer.wpe.weight\n",
      "     from: torch.Size([1024, 768])\n",
      "       to: torch.Size([1024, 768])\n",
      "   Loading: transformer.h.0.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.0.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.0.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.0.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.0.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.0.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.0.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.0.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.1.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.1.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.1.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.1.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.1.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.1.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.1.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.2.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.2.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.2.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.2.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.2.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.2.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.2.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.3.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.3.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.3.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.3.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.3.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.3.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.3.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.4.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.4.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.4.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.4.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.4.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.4.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.4.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.5.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.5.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.5.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.5.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.5.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.5.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.5.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.6.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.6.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.6.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.6.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.6.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.6.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.6.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.7.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.7.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.7.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.7.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.7.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.7.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.7.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.8.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.8.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.8.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.8.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.8.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.8.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.8.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.9.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.9.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.9.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.9.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.9.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.9.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.9.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.10.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.10.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.10.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.10.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.10.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.10.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.10.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_1.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_1.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.attn.c_attn.weight\n",
      "     from: torch.Size([2304, 768])\n",
      "       to: torch.Size([2304, 768])\n",
      "   Loading: transformer.h.11.attn.c_attn.bias\n",
      "     from: torch.Size([2304])\n",
      "       to: torch.Size([2304])\n",
      "   Loading: transformer.h.11.attn.c_proj.weight\n",
      "     from: torch.Size([768, 768])\n",
      "       to: torch.Size([768, 768])\n",
      "   Loading: transformer.h.11.attn.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_2.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.ln_2.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.h.11.mlp.c_fc.weight\n",
      "     from: torch.Size([3072, 768])\n",
      "       to: torch.Size([3072, 768])\n",
      "   Loading: transformer.h.11.mlp.c_fc.bias\n",
      "     from: torch.Size([3072])\n",
      "       to: torch.Size([3072])\n",
      "   Loading: transformer.h.11.mlp.c_proj.weight\n",
      "     from: torch.Size([768, 3072])\n",
      "       to: torch.Size([768, 3072])\n",
      "   Loading: transformer.h.11.mlp.c_proj.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.ln_f.weight\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: transformer.ln_f.bias\n",
      "     from: torch.Size([768])\n",
      "       to: torch.Size([768])\n",
      "   Loading: lm_head.weight\n",
      "     from: torch.Size([50304, 768])\n",
      "       to: torch.Size([50304, 768])\n"
     ]
    }
   ],
   "source": [
    "from gpt2 import GPT\n",
    "\n",
    "model = GPT.from_pretrained(\"models/shakespear_500.pth\", local=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> KING RICHARD II:\n",
      "A sun hath fly the good shed a thousand, or I may add;\n",
      "DUKE VINCENTIO\n",
      "> KING RICHARD II:\n",
      "My prince, my lord;\n",
      "No think nothing on her, be even by this set and bring he must be\n",
      "> KING RICHARD II:\n",
      "Rway with me not easily under the\n",
      "ARCH:\n",
      "As I would the\n",
      "My lord,\n",
      "KING R\n",
      "> KING RICHARD II:\n",
      "T thou, my tent I should be deaf.\n",
      "\n",
      "ISABRAHAM:\n",
      "For I may be but\n",
      "> KING RICHARD II: it from me 'twere well,\n",
      "KING RICHARD III:\n",
      "So. Therefore, bring me words,\n"
     ]
    }
   ],
   "source": [
    "model.generate(\"KING RICHARD II:\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mil",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
